package spark_core.operate_transform;

import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

/**
 * @author shihb
 * @date 2020/1/7 18:54
 * 多个Rdd交互demo
 */
public class MultiValueDemo {

  public static void main(String[] args) {
    //local模式,创建SparkConf对象设定spark的部署环境
    SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("mark rdd");
    //创建spark上下文对象（这边是java上下文）
    JavaSparkContext sc = new JavaSparkContext(sparkConf);

    //kv多流
    JavaPairRDD pairRdd1 = sc.parallelizePairs(Arrays
            .asList(new Tuple2(1, "aa"), new Tuple2(2, "bb"), new Tuple2(3, "cc"), new Tuple2(4, "dd")),
        1);
    JavaPairRDD pairRdd2 = sc.parallelizePairs(Arrays
            .asList(new Tuple2(1, 5), new Tuple2(2, 6), new Tuple2(3, 7), new Tuple2(4, 8)),
        1);

    JavaPairRDD joinRdd = pairRdd1.join(pairRdd2, 1);
    joinRdd.glom().collect().forEach(System.out::println);
//  [(4,(dd,8)), (1,(aa,5)), (3,(cc,7)), (2,(bb,6))]



    //单值多流
    JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4),1);
    JavaRDD<Integer> rdd2 = sc.parallelize(Arrays.asList(4, 5, 6, 7),1);
    //合并
    JavaRDD<Integer> unionRdd = rdd1.union(rdd2);
    unionRdd.glom().collect().forEach(System.out::println);
//  [1, 2, 3, 4]
//  [4, 5, 6, 7]
    //rdd1去除两个rdd2重复的数据
    JavaRDD<Integer> subtractRdd = rdd1.subtract(rdd2);
    subtractRdd.glom().collect().forEach(System.out::println);
//    [1, 2, 3]
    //相交
    JavaRDD<Integer> intersectionRdd = rdd1.intersection(rdd2);
    intersectionRdd.glom().collect().forEach(System.out::println);
    //[4]
    //笛卡尔积
    JavaPairRDD<Integer, Integer> cartesianRdd = rdd1.cartesian(rdd2);
    cartesianRdd.glom().collect().forEach(System.out::println);
    //[(1,4), (1,5), (1,6), (1,7), (2,4), (2,5), (2,6), (2,7), (3,4), (3,5), (3,6), (3,7), (4,4), (4,5), (4,6), (4,7)]

    //
    //停止
    sc.stop();
  }

}
